###############################
### DO NOT CHANGE THIS CELL ###
###############################
# This is cell which sets up some of the modules you might need
# Please do not change the cell or import any additional packages.

import sys
import warnings

import matplotlib
import numpy as np
import pandas as pd
import plotly.io as pio
from matplotlib import pyplot as plt
from sklearn.datasets import load_breast_cancer, load_diabetes, load_iris
from sklearn.feature_extraction import text
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, mean_squared_error
from sklearn.model_selection import train_test_split

print("Version information")

print("python: {}".format(sys.version))
print("matplotlib: {}".format(matplotlib.__version__))
print("numpy: {}".format(np.__version__))

warnings.filterwarnings("ignore")

%matplotlib inline
%load_ext autoreload
%autoreload 2

STUDENT_VERSION = 0
EO_TEXT, EO_FONT, EO_COLOR = (
    "TA VERSION",
    "Chalkduster",
    "gray",
)
EO_ALPHA, EO_SIZE, EO_ROT = 0.7, 90, 40

pio.renderers.default = "jupyterlab"

Version information
python: 3.11.8 (main, Feb 26 2024, 15:36:12) [Clang 14.0.6 ]
matplotlib: 3.8.0
numpy: 1.26.4

###############################
### DO NOT CHANGE THIS CELL ###
###############################
# load Image
image = plt.imread("./data/hw3_image_compression.jpeg") / 255
# plot image
fig = plt.figure(figsize=(10, 10))
plt.imshow(image)

<matplotlib.image.AxesImage at 0x16183ac10>

###############################
### DO NOT CHANGE THIS CELL ###
###############################


def rgb2gray(rgb):
    return np.dot(rgb[..., :3], [0.299, 0.587, 0.114])


fig = plt.figure(figsize=(10, 10))


# plot several images
plt.imshow(rgb2gray(image), cmap=plt.cm.bone)

<matplotlib.image.AxesImage at 0x16c33a910>

###############################
### DO NOT CHANGE THIS CELL ###
###############################

from utilities.localtests import TestImgCompression

unittest_ic = TestImgCompression()
unittest_ic.test_svd_bw()
unittest_ic.test_compress_bw()
unittest_ic.test_rebuild_svd_bw()
unittest_ic.test_compression_ratio_bw()
unittest_ic.test_recovered_variance_proportion_bw()

UnitTest passed successfully for "SVD calculation - black and white images"!
UnitTest passed successfully for "Image compression - black and white images"!
UnitTest passed successfully for "SVD reconstruction - black and white images"!
UnitTest passed successfully for "Compression ratio - black and white images"!
UnitTest passed successfully for "Recovered variance proportion - black and white images"!

###############################
### DO NOT CHANGE THIS CELL ###
###############################

from utilities.localtests import TestImgCompression

unittest_ic = TestImgCompression()

unittest_ic.test_svd_color()
unittest_ic.test_compress_color()
unittest_ic.test_rebuild_svd_color()
unittest_ic.test_compression_ratio_color()
unittest_ic.test_recovered_variance_proportion_color()

UnitTest passed successfully for "SVD calculation - color images"!
UnitTest passed successfully for "Image compression - color images"!
UnitTest passed successfully for "SVD reconstruction - color images"!
UnitTest passed successfully for "Compression ratio - color images"!
UnitTest passed successfully for "Recovered variance proportion - color images"!

###############################
### DO NOT CHANGE THIS CELL ###
###############################
from imgcompression import ImgCompression

imcompression = ImgCompression()
bw_image = rgb2gray(image)
U, S, V = imcompression.svd(bw_image)
component_num = [1, 2, 4, 8, 16, 32, 64, 128, 256]

fig = plt.figure(figsize=(18, 18))

# plot several images
i = 0
for k in component_num:
    U_compressed, S_compressed, V_compressed = imcompression.compress(U, S, V, k)
    img_rebuild = imcompression.rebuild_svd(U_compressed, S_compressed, V_compressed)
    c = np.around(imcompression.compression_ratio(bw_image, k), 4)
    r = np.around(imcompression.recovered_variance_proportion(S, k), 3)
    ax = fig.add_subplot(3, 3, i + 1, xticks=[], yticks=[])
    ax.imshow(img_rebuild, cmap=plt.cm.bone)
    ax.set_title(f"{k} Components")
    ax.set_xlabel(f"Compression: {c},\nRecovered Variance: {r}")
    i = i + 1

###############################
### DO NOT CHANGE THIS CELL ###
###############################
from imgcompression import ImgCompression

imcompression = ImgCompression()
bw_image = rgb2gray(image)
U, S, V = imcompression.svd(bw_image)

component_num = [1, 2, 4, 8, 16, 32, 64, 128, 256]

# Compare memory savings for BW image
for k in component_num:
    og_bytes, comp_bytes, savings = imcompression.memory_savings(bw_image, U, S, V, k)
    comp_ratio = og_bytes / comp_bytes
    og_bytes = imcompression.nbytes_to_string(og_bytes)
    comp_bytes = imcompression.nbytes_to_string(comp_bytes)
    savings = imcompression.nbytes_to_string(savings)
    print(
        f"{k} components: Original Image: {og_bytes} -> Compressed Image: {comp_bytes}, Savings: {savings}, Compression Ratio {comp_ratio:.1f}:1"
    )

1 components: Original Image: 10.986 MB -> Compressed Image: 18.758 KB, Savings: 10.968 MB, Compression Ratio 599.8:1
2 components: Original Image: 10.986 MB -> Compressed Image: 37.516 KB, Savings: 10.95 MB, Compression Ratio 299.9:1
4 components: Original Image: 10.986 MB -> Compressed Image: 75.031 KB, Savings: 10.913 MB, Compression Ratio 149.9:1
8 components: Original Image: 10.986 MB -> Compressed Image: 150.062 KB, Savings: 10.84 MB, Compression Ratio 75.0:1
16 components: Original Image: 10.986 MB -> Compressed Image: 300.125 KB, Savings: 10.693 MB, Compression Ratio 37.5:1
32 components: Original Image: 10.986 MB -> Compressed Image: 600.25 KB, Savings: 10.4 MB, Compression Ratio 18.7:1
64 components: Original Image: 10.986 MB -> Compressed Image: 1.172 MB, Savings: 9.814 MB, Compression Ratio 9.4:1
128 components: Original Image: 10.986 MB -> Compressed Image: 2.345 MB, Savings: 8.642 MB, Compression Ratio 4.7:1
256 components: Original Image: 10.986 MB -> Compressed Image: 4.689 MB, Savings: 6.297 MB, Compression Ratio 2.3:1

###############################
### DO NOT CHANGE THIS CELL ###
###############################
from imgcompression import ImgCompression

imcompression = ImgCompression()
image_rolled = np.moveaxis(image, -1, 0)
U, S, V = imcompression.svd(image_rolled)

component_num = [1, 2, 4, 8, 16, 32, 64, 128, 256]

fig = plt.figure(figsize=(18, 18))

# plot several images
i = 0
for k in component_num:
    U_compressed, S_compressed, V_compressed = imcompression.compress(U, S, V, k)
    img_rebuild = np.clip(
        imcompression.rebuild_svd(U_compressed, S_compressed, V_compressed), 0, 1
    )
    img_rebuild = np.moveaxis(img_rebuild, 0, -1)
    c = np.around(imcompression.compression_ratio(image_rolled, k), 4)
    r = np.around(imcompression.recovered_variance_proportion(S, k), 3)
    ax = fig.add_subplot(3, 3, i + 1, xticks=[], yticks=[])
    ax.imshow(img_rebuild)
    ax.set_title(f"{k} Components")
    ax.set_xlabel(
        f"Compression: {np.around(c,4)},\nRecovered Variance:  R: {r[0]}  G: {r[1]}  B: {r[2]}"
    )
    i = i + 1

###############################
### DO NOT CHANGE THIS CELL ###
###############################
from imgcompression import ImgCompression

imcompression = ImgCompression()
U, S, V = imcompression.svd(image_rolled)

component_num = [1, 2, 4, 8, 16, 32, 64, 128, 256]

# Compare the memory savings of the color image
i = 0
for k in component_num:
    og_bytes, comp_bytes, savings = imcompression.memory_savings(
        image_rolled, U, S, V, k
    )
    comp_ratio = og_bytes / comp_bytes
    og_bytes = imcompression.nbytes_to_string(og_bytes)
    comp_bytes = imcompression.nbytes_to_string(comp_bytes)
    savings = imcompression.nbytes_to_string(savings)
    print(
        f"{k} components: Original Image: {og_bytes} -> Compressed Image: {comp_bytes}, Savings: {savings}, Compression Ratio {comp_ratio:.1f}:1"
    )

1 components: Original Image: 32.959 MB -> Compressed Image: 56.273 KB, Savings: 32.904 MB, Compression Ratio 599.8:1
2 components: Original Image: 32.959 MB -> Compressed Image: 112.547 KB, Savings: 32.849 MB, Compression Ratio 299.9:1
4 components: Original Image: 32.959 MB -> Compressed Image: 225.094 KB, Savings: 32.739 MB, Compression Ratio 149.9:1
8 components: Original Image: 32.959 MB -> Compressed Image: 450.188 KB, Savings: 32.519 MB, Compression Ratio 75.0:1
16 components: Original Image: 32.959 MB -> Compressed Image: 900.375 KB, Savings: 32.08 MB, Compression Ratio 37.5:1
32 components: Original Image: 32.959 MB -> Compressed Image: 1.759 MB, Savings: 31.2 MB, Compression Ratio 18.7:1
64 components: Original Image: 32.959 MB -> Compressed Image: 3.517 MB, Savings: 29.442 MB, Compression Ratio 9.4:1
128 components: Original Image: 32.959 MB -> Compressed Image: 7.034 MB, Savings: 25.925 MB, Compression Ratio 4.7:1
256 components: Original Image: 32.959 MB -> Compressed Image: 14.068 MB, Savings: 18.891 MB, Compression Ratio 2.3:1

###############################
### DO NOT CHANGE THIS CELL ###
###############################

from utilities.localtests import TestPCA

unittest_pca = TestPCA()
unittest_pca.test_pca()
unittest_pca.test_transform()
unittest_pca.test_transform_rv()

UnitTest passed successfully for "PCA fit"!
UnitTest passed successfully for "PCA transform"!
UnitTest passed successfully for "PCA transform with recovered variance"!

###############################
### DO NOT CHANGE THIS CELL ###
###############################

# Use PCA for visualization of iris dataset

from pca import PCA

iris_data = load_iris(return_X_y=True)

X = iris_data[0]
y = iris_data[1]

fig_title = "Iris Dataset with Dimensionality Reduction"

pca = PCA()
pca.fit(X)
pca.visualize(X, y, fig_title)

###############################
### DO NOT CHANGE THIS CELL ###
###############################

X = np.load("./data/smallflat_64.npy")
y = np.load("./data/masked_labels.npy").astype("int")
i = 0
fig = plt.figure(figsize=(18, 18))
for idx in [0, 1, 2, 150, 151, 152]:
    ax = fig.add_subplot(6, 6, i + 1, xticks=[], yticks=[])
    image = (
        np.rot90(X[idx].reshape(64, 64), k=1)
        if idx % 2 == 1 and idx < 150
        else X[idx].reshape(64, 64)
    )
    m_status = "Unmasked" if idx < 150 else "Masked"
    ax.imshow(image, cmap="gray")
    ax.set_title(f"{m_status} Image at i = {idx}")
    i += 1

###############################
### DO NOT CHANGE THIS CELL ###
###############################
# Use PCA for visualization of masked and unmasked images

X = np.load("./data/smallflat_64.npy")
y = np.load("./data/masked_labels.npy")

fig_title = "Facemask Dataset Visualization with Dimensionality Reduction"

pca = PCA()
pca.fit(X)
pca.visualize(X, y, fig_title)


print(
    "*In this plot, the 0 points are unmasked images and the 1 points are masked images."
)

*In this plot, the 0 points are unmasked images and the 1 points are masked images.

###############################
### DO NOT CHANGE THIS CELL ###
###############################

X = np.load("./data/emotion_features.npy")
y = np.load("./data/emotion_labels.npy").astype("int")
i = 0
fig = plt.figure(figsize=(18, 18))
for idx in [0, 1, 2, 150, 151, 152]:
    ax = fig.add_subplot(6, 6, i + 1, xticks=[], yticks=[])
    image = (
        np.rot90(X[idx].reshape(64, 64), k=1)
        if idx % 2 == 1 and idx < 150
        else X[idx].reshape(64, 64)
    )
    m_status = "Unmasked" if idx < 150 else "Masked"
    ax.imshow(image, cmap="gray")
    m_status = "Sad" if idx < 150 else "Happy"
    ax.set_title(f"{m_status} Image at i = {idx}")
    i += 1

###############################
### DO NOT CHANGE THIS CELL ###
###############################

X = np.load("./data/emotion_features.npy")
y = np.load("./data/emotion_labels.npy").astype("int")

print("Not Graded - Data shape before PCA ", X.shape)

pca = PCA()
pca.fit(X)

X_pca = pca.transform_rv(X, retained_variance=0.99)

print("Not Graded - Data shape with PCA ", X_pca.shape)

Not Graded - Data shape before PCA  (600, 4096)
Not Graded - Data shape with PCA  (600, 150)

###############################
### DO NOT CHANGE THIS CELL ###
###############################
# Train, test splits
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.3, stratify=y, random_state=42
)

# Use logistic regression to predict classes for test set
clf = LogisticRegression()
clf.fit(X_train, y_train)
preds = clf.predict_proba(X_test)
print(
    "Not Graded - Accuracy before PCA: {:.5f}".format(
        accuracy_score(y_test, preds.argmax(axis=1))
    )
)

Not Graded - Accuracy before PCA: 0.95000

###############################
### DO NOT CHANGE THIS CELL ###
###############################
# Train, test splits
X_train, X_test, y_train, y_test = train_test_split(
    X_pca, y, test_size=0.3, stratify=y, random_state=42
)

# Use logistic regression to predict classes for test set
clf = LogisticRegression()
clf.fit(X_train, y_train)
preds = clf.predict_proba(X_test)
print(
    "Not Graded - Accuracy after PCA: {:.5f}".format(
        accuracy_score(y_test, preds.argmax(axis=1))
    )
)

Not Graded - Accuracy after PCA: 0.95556

###############################
### DO NOT CHANGE THIS CELL ###
###############################
from sklearn.linear_model import RidgeCV


def apply_regression(X_train, y_train, X_test):
    ridge = RidgeCV(alphas=[1e-3, 1e-2, 1e-1, 1])
    clf = ridge.fit(X_train, y_train)
    y_pred = ridge.predict(X_test)

    return y_pred

###############################
### DO NOT CHANGE THIS CELL ###
###############################
# load the dataset
diabetes = load_diabetes()
X = diabetes.data
y = diabetes.target

print(X.shape, y.shape)

pca = PCA()
pca.fit(X)

X_pca = pca.transform_rv(X, retained_variance=0.9)
print("Not Graded - data shape with PCA ", X_pca.shape)

(442, 10) (442,)
Not Graded - data shape with PCA  (442, 7)

###############################
### DO NOT CHANGE THIS CELL ###
###############################
# Train, test splits
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.3, random_state=42
)

# Ridge regression without PCA
y_pred = apply_regression(X_train, y_train, X_test)

# calculate RMSE
rmse_score = np.sqrt(mean_squared_error(y_pred, y_test))
print(
    "Not Graded - RMSE score using Ridge Regression before PCA: {:.5}".format(
        rmse_score
    )
)

Not Graded - RMSE score using Ridge Regression before PCA: 53.101

###############################
### DO NOT CHANGE THIS CELL ###
###############################
# Ridge regression with PCA
X_train, X_test, y_train, y_test = train_test_split(
    X_pca, y, test_size=0.3, random_state=42
)

# use Ridge Regression for getting predicted labels
y_pred = apply_regression(X_train, y_train, X_test)

# calculate RMSE
rmse_score = np.sqrt(mean_squared_error(y_pred, y_test))
print(
    "Not Graded - RMSE score using Ridge Regression after PCA: {:.5}".format(rmse_score)
)

Not Graded - RMSE score using Ridge Regression after PCA: 53.024

###############################
### DO NOT CHANGE THIS CELL ###
###############################

from utilities.localtests import TestRegression

unittest_reg = TestRegression()
unittest_reg.test_rmse()
unittest_reg.test_construct_polynomial_feats()
unittest_reg.test_predict()

UnitTest passed successfully for "RMSE"!
UnitTest passed successfully for "Polynomial feature construction"!
UnitTest passed successfully for "Linear regression prediction"!

###############################
### DO NOT CHANGE THIS CELL ###
###############################

from utilities.localtests import TestRegression

unittest_reg = TestRegression()
unittest_reg.test_linear_fit_closed()

UnitTest passed successfully for "Closed form linear regression"!

###############################
### DO NOT CHANGE THIS CELL ###
###############################

from utilities.localtests import TestRegression

unittest_reg = TestRegression()
unittest_reg.test_ridge_fit_closed()
unittest_reg.test_ridge_cross_validation()

UnitTest passed successfully for "Closed form ridge regression"!
UnitTest passed successfully for "Ridge regression cross validation"!

###############################
### DO NOT CHANGE THIS CELL ###
###############################

from utilities.localtests import TestRegression

unittest_reg = TestRegression()
unittest_reg.test_linear_fit_GD()
unittest_reg.test_linear_fit_SGD()
unittest_reg.test_ridge_fit_GD()
unittest_reg.test_ridge_fit_SGD()

UnitTest passed successfully for "Gradient descent linear regression"!
UnitTest passed successfully for "Stochastic gradient descent linear regression"!
UnitTest passed successfully for "Gradient descent ridge regression"!
UnitTest passed successfully for "Stochastic gradient descent ridge regression"!

###############################
### DO NOT CHANGE THIS CELL ###
###############################

from plotter import Plotter
from regression import Regression

###############################
### DO NOT CHANGE THIS CELL ###
###############################

# Generate a sample regression dataset with polynomial features
# using the student's regression implementation.

POLY_DEGREE = 5

reg = Regression()
plotter = Plotter(regularization=reg, poly_degree=POLY_DEGREE)

x_all, y_all, p, x_all_feat = plotter.create_data()

x_all: 700 (rows/samples) 2 (columns/features)
y_all: 700 (rows/samples) 1 (columns/features)

###############################
### DO NOT CHANGE THIS CELL ###
###############################

# Visualize simulated regression data

plotter.plot_all_data(x_all, y_all, p)

###############################
### DO NOT CHANGE THIS CELL ###
###############################

xtrain, ytrain, xtest, ytest, train_indices, test_indices = plotter.split_data(
    x_all, y_all
)

plotter.plot_split_data(xtrain, xtest, ytrain, ytest)

###############################
### DO NOT CHANGE THIS CELL ###
###############################

# Required for both Grad and Undergrad

weight = reg.linear_fit_closed(x_all_feat[train_indices], y_all[train_indices])
y_test_pred = reg.predict(x_all_feat[test_indices], weight)
test_rmse = reg.rmse(y_test_pred, y_all[test_indices])
y_pred = reg.predict(x_all_feat, weight)
print("Linear (closed) RMSE: %.4f" % test_rmse)

plotter.plot_linear_closed(xtrain, xtest, ytrain, ytest, x_all, y_pred)

Linear (closed) RMSE: 1.0273

###############################
### DO NOT CHANGE THIS CELL ###
###############################

# Required for Grad Only
# This cell may take more than 1 minute
weight, _ = reg.linear_fit_GD(
    x_all_feat[train_indices], y_all[train_indices], epochs=50000, learning_rate=1e-8
)
y_test_pred = reg.predict(x_all_feat[test_indices], weight)
test_rmse = reg.rmse(y_test_pred, y_all[test_indices])
print("Linear (GD) RMSE: %.4f" % test_rmse)
y_pred = reg.predict(x_all_feat, weight)
y_pred = np.reshape(y_pred, (y_pred.size,))

plotter.plot_linear_gd(xtrain, xtest, ytrain, ytest, x_all, y_pred)

Linear (GD) RMSE: 3.1861

###############################
### DO NOT CHANGE THIS CELL ###
###############################

# Required for Grad Only
# This cell may take more than 1 minute

learning_rates = [1e-8, 1e-6, 1e-4]
weights = np.zeros((3, POLY_DEGREE**2 + 2))

for ii in range(len(learning_rates)):
    weights[ii, :] = reg.linear_fit_GD(
        x_all_feat[train_indices],
        y_all[train_indices],
        epochs=50000,
        learning_rate=learning_rates[ii],
    )[0].ravel()
    y_test_pred = reg.predict(
        x_all_feat[test_indices], weights[ii, :].reshape((POLY_DEGREE**2 + 2, 1))
    )
    test_rmse = reg.rmse(y_test_pred, y_all[test_indices])
    print("Linear (GD) RMSE: %.4f (learning_rate=%s)" % (test_rmse, learning_rates[ii]))

plotter.plot_linear_gd_tuninglr(
    xtrain, xtest, ytrain, ytest, x_all, x_all_feat, learning_rates, weights
)

Linear (GD) RMSE: 3.1861 (learning_rate=1e-08)
Linear (GD) RMSE: 2.2901 (learning_rate=1e-06)
Linear (GD) RMSE: 1.1099 (learning_rate=0.0001)

###############################
### DO NOT CHANGE THIS CELL ###
###############################
rng = np.random.RandomState(seed=3)
y_all_noisy = np.dot(x_all_feat, np.zeros((POLY_DEGREE**2 + 2, 1))) + rng.randn(
    x_all_feat.shape[0], 1
)
sub_train = train_indices[10:20]

###############################
### DO NOT CHANGE THIS CELL ###
###############################

# Required for both Grad and Undergrad

weight = reg.linear_fit_closed(x_all_feat[sub_train], y_all_noisy[sub_train])
y_pred = reg.predict(x_all_feat, weight)
y_test_pred = reg.predict(x_all_feat[test_indices], weight)
test_rmse = reg.rmse(y_test_pred, y_all_noisy[test_indices])
print("Linear (closed) 10 Samples RMSE: %.4f" % test_rmse)

plotter.plot_10_samples(
    x_all, y_all_noisy, sub_train, y_pred, title="Linear Regression (Closed)"
)

Linear (closed) 10 Samples RMSE: 1816.3828

###############################
### DO NOT CHANGE THIS CELL ###
###############################
# Required for both Grad and Undergrad
weight = reg.ridge_fit_closed(
    x_all_feat[sub_train], y_all_noisy[sub_train], c_lambda=10
)
y_pred = reg.predict(x_all_feat, weight)
y_test_pred = reg.predict(x_all_feat[test_indices], weight)
test_rmse = reg.rmse(y_test_pred, y_all_noisy[test_indices])
print("Ridge Regression (closed) RMSE: %.4f" % test_rmse)

plotter.plot_10_samples(
    x_all, y_all_noisy, sub_train, y_pred, title="Ridge Regression (Closed)"
)

Ridge Regression (closed) RMSE: 1.1193

###############################
### DO NOT CHANGE THIS CELL ###
###############################

# Required for Grad Only

weight, _ = reg.ridge_fit_GD(
    x_all_feat[sub_train], y_all_noisy[sub_train], c_lambda=20, learning_rate=1e-5
)
y_pred = reg.predict(x_all_feat, weight)
y_test_pred = reg.predict(x_all_feat[test_indices], weight)
test_rmse = reg.rmse(y_test_pred, y_all_noisy[test_indices])
print("Ridge Regression (GD) RMSE: %.4f" % test_rmse)

plotter.plot_10_samples(
    x_all, y_all_noisy, sub_train, y_pred, title="Ridge Regression (GD)"
)

Ridge Regression (GD) RMSE: 1.0413

###############################
### DO NOT CHANGE THIS CELL ###
###############################

# Required for Grad Only

weight, _ = reg.ridge_fit_SGD(
    x_all_feat[sub_train], y_all_noisy[sub_train], c_lambda=20, learning_rate=1e-5
)
y_pred = reg.predict(x_all_feat, weight)
y_test_pred = reg.predict(x_all_feat[test_indices], weight)
test_rmse = reg.rmse(y_test_pred, y_all_noisy[test_indices])
print("Ridge Regression (SGD) RMSE: %.4f" % test_rmse)

plotter.plot_10_samples(
    x_all, y_all_noisy, sub_train, y_pred, title="Ridge Regression (SGD)"
)

Ridge Regression (SGD) RMSE: 1.0410

###############################
### DO NOT CHANGE THIS CELL ###
###############################
lambda_list = [0.0001, 0.001, 0.1, 1, 5, 10, 50, 100, 1000, 10000]
kfold = 5

best_lambda, best_error, error_list = reg.hyperparameter_search(
    x_all_feat[train_indices], y_all[train_indices], lambda_list, kfold
)
for lm, err in zip(lambda_list, error_list):
    print("Lambda: %.4f" % lm, "RMSE: %.6f" % err)

print("Best Lambda: %.4f" % best_lambda)
weight = reg.ridge_fit_closed(
    x_all_feat[train_indices], y_all_noisy[train_indices], c_lambda=best_lambda
)
y_test_pred = reg.predict(x_all_feat[test_indices], weight)
test_rmse = reg.rmse(y_test_pred, y_all_noisy[test_indices])
print("Best Test RMSE: %.4f" % test_rmse)

Lambda: 0.0001 RMSE: 0.986072
Lambda: 0.0010 RMSE: 0.987209
Lambda: 0.1000 RMSE: 0.989441
Lambda: 1.0000 RMSE: 0.987945
Lambda: 5.0000 RMSE: 0.986684
Lambda: 10.0000 RMSE: 0.986821
Lambda: 50.0000 RMSE: 0.989110
Lambda: 100.0000 RMSE: 0.994419
Lambda: 1000.0000 RMSE: 1.289583
Lambda: 10000.0000 RMSE: 2.544557
Best Lambda: 0.0001
Best Test RMSE: 1.0528

###############################
### DO NOT CHANGE THIS CELL ###
###############################

from utilities.localtests import TestLogisticRegression

unittest_lr = TestLogisticRegression()
unittest_lr.test_sigmoid()
unittest_lr.test_bias_augment()
unittest_lr.test_loss()
unittest_lr.test_predict_probs()
unittest_lr.test_predict_labels()
unittest_lr.test_loss()
unittest_lr.test_accuracy()
unittest_lr.test_evaluate()
unittest_lr.test_fit()

UnitTest passed successfully for "Logistic Regression sigmoid"!
UnitTest passed successfully for "Logistic Regression bias_augment"!
UnitTest passed successfully for "Logistic Regression loss"!
UnitTest passed successfully for "Logistic Regression predict_probs"!
UnitTest passed successfully for "Logistic Regression predict_labels"!
UnitTest passed successfully for "Logistic Regression loss"!
UnitTest passed successfully for "Logistic Regression accuracy"!
UnitTest passed successfully for "Logistic Regression evaluate"!
Epoch 0:
	train loss: 0.675	train acc: 0.7
	val loss:   0.675	val acc:   0.7
UnitTest passed successfully for "Logistic Regression fit"!

###############################
### DO NOT CHANGE THIS CELL ###
###############################

from logistic_regression import LogisticRegression as LogReg

###############################
### DO NOT CHANGE THIS CELL ###
###############################

news_data = pd.read_csv("./data/news-data.csv", encoding="cp437", header=None)

class_to_label_mappings = {"negative": 0, "positive": 1}

label_to_class_mappings = {0: "negative", 1: "positive"}

news_data.columns = ["Sentiment", "News"]
news_data.drop_duplicates(inplace=True)

news_data = news_data[news_data.Sentiment != "neutral"]

news_data["Sentiment"] = news_data["Sentiment"].map(class_to_label_mappings)

vectorizer = text.CountVectorizer(stop_words="english")

X = news_data["News"].values
y = news_data["Sentiment"].values.reshape(-1, 1)

RANDOM_SEED = 5
BOW = vectorizer.fit_transform(X).toarray()
indices = np.arange(len(news_data))
X_train, X_test, y_train, y_test, indices_train, indices_test = train_test_split(
    BOW, y, indices, test_size=0.2, random_state=RANDOM_SEED
)

###############################
### DO NOT CHANGE THIS CELL ###
###############################

model = LogReg()
lr = 0.05
epochs = 10000
theta = model.fit(X_train, y_train, X_test, y_test, lr, epochs)

Epoch 0:
	train loss: 0.69	train acc: 0.7
	val loss:   0.691	val acc:   0.665
Epoch 1000:
	train loss: 0.436	train acc: 0.794
	val loss:   0.532	val acc:   0.701
Epoch 2000:
	train loss: 0.364	train acc: 0.846
	val loss:   0.484	val acc:   0.746
Epoch 3000:
	train loss: 0.318	train acc: 0.873
	val loss:   0.456	val acc:   0.761
Epoch 4000:
	train loss: 0.286	train acc: 0.896
	val loss:   0.438	val acc:   0.772
Epoch 5000:
	train loss: 0.262	train acc: 0.914
	val loss:   0.425	val acc:   0.782
Epoch 6000:
	train loss: 0.242	train acc: 0.926
	val loss:   0.416	val acc:   0.789
Epoch 7000:
	train loss: 0.226	train acc: 0.933
	val loss:   0.409	val acc:   0.797
Epoch 8000:
	train loss: 0.212	train acc: 0.943
	val loss:   0.404	val acc:   0.802
Epoch 9000:
	train loss: 0.2	train acc: 0.95
	val loss:   0.4	val acc:   0.799

###############################
### DO NOT CHANGE THIS CELL ###
###############################

test_loss, test_acc = model.evaluate(X_test, y_test, theta)
print(f"Test Dataset Accuracy: {round(test_acc, 3)}")

Test Dataset Accuracy: 0.807

###############################
### DO NOT CHANGE THIS CELL ###
###############################

model.plot_loss()

###############################
### DO NOT CHANGE THIS CELL ###
###############################

model.plot_accuracy()

###############################
### DO NOT CHANGE THIS CELL ###
###############################

num_samples = 10
for i in range(10):
    rand_index = np.random.randint(0, len(X_test))
    x_test = np.reshape(X_test[rand_index], (1, X_test.shape[1]))
    prob = model.predict_probs(model.bias_augment(x_test), theta)
    pred = model.predict_labels(prob)
    print(f"Input News: {X[indices_test[rand_index]]}\n")
    print(f"Predicted Sentiment: {label_to_class_mappings[pred[0][0]]}")
    print(f"Actual Sentiment: {label_to_class_mappings[y_test[rand_index][0]]}\n")

Input News: Cargotec Corporation , Press Release , August 26 , 2008 at 10 a.m. Finnish time Cargotec 's MacGREGOR business area providing marine cargo handling and offshore load handling solutions has received significant offshore crane retrofit order .

Predicted Sentiment: positive
Actual Sentiment: positive

Input News: Martela said plans to expand its recycled furniture business elsewhere in Finland , too .

Predicted Sentiment: positive
Actual Sentiment: positive

Input News: Ruukki Romania , the local arm of Finnish metal producer Ruukki , increased its capital by 900,000 euro ( $ 1.14 mln ) through cash contribution , it was reported on September 19 , 2006 .

Predicted Sentiment: positive
Actual Sentiment: positive

Input News: : Lietuvos Respublikos sveikatos apsaugos ministerija has awarded contract to UAB `` AFFECTO LIETUVA '' for financial systems software package .

Predicted Sentiment: positive
Actual Sentiment: positive

Input News: In January-June 2010 , diluted loss per share stood at EUR0 .3 versus EUR0 .1 in the first half of 2009 .

Predicted Sentiment: positive
Actual Sentiment: negative

Input News: Componenta 's objective with this agreement is to increase business on its existing production lines .

Predicted Sentiment: positive
Actual Sentiment: positive

Input News: In contrast , the company 's net loss for the third quarter of 2009 contracted to EUR 76 million from EUR 256 million for the corresponding period a year ago .

Predicted Sentiment: positive
Actual Sentiment: positive

Input News: The Finnish supplier of BSS-OSS and VAS for telecom operators , Tecnotree , has received expansion orders worth a total US$ 7.3 mn for its convergent charging and next generation messaging solutions in Latin America , the company announced without specifying which operators had placed the orders .

Predicted Sentiment: positive
Actual Sentiment: positive

Input News: The company said that the fall in turnover had been planned .

Predicted Sentiment: positive
Actual Sentiment: negative

Input News: According to Arokarhu , some of the purchases that had been scanned into the cash register computer disappeared when the total sum key was pressed .

Predicted Sentiment: negative
Actual Sentiment: negative

from slope import Slope

###############################
### DO NOT CHANGE THIS CELL ###
###############################
x = np.arange(0, 1.02, 0.02).reshape(-1, 1)

y = 5 * np.arange(0, 1.02, 0.02).reshape(-1, 1)

print("Slope of first principal component", Slope.pca_slope(x, y))

print("Slope of best linear fit", Slope.lr_slope(x, y))

fig = plt.figure()
plt.scatter(x, y)
plt.xlabel("x")
plt.ylabel("y")

plt.show()

Slope of first principal component 0.9805806756909202
Slope of best linear fit [5.]

###############################
### DO NOT CHANGE THIS CELL ###
###############################
base = np.arange(0.001, 1.001, 0.001).reshape(-1, 1)
c = 0.5
X = base
y = 5 * base + np.random.normal(loc=[0], scale=c, size=base.shape)

fig = plt.figure()
plt.scatter(X, y)
plt.xlabel("x")
plt.ylabel("y")

plt.show()

###############################
### DO NOT CHANGE THIS CELL ###
###############################
pca_slope_values = []
linreg_slope_values = []
c_values = []
s_idx = 0

for i in range(30):
    for c in np.arange(0, 1.05, 0.05):
        # Calculate pca_slope_value (psv) and lr_slope_value (lsv)
        psv, lsv = Slope.addNoise(c, x_noise=False, seed=s_idx)

        # Append pca and lr slope values to list for plot function
        pca_slope_values.append(psv)
        linreg_slope_values.append(lsv)

        # Append c value to list for plot function
        c_values.append(c)

        # Increment random seed index
        s_idx += 1

fig = plt.figure()
plt.scatter(c_values, pca_slope_values, c="r")
plt.scatter(c_values, linreg_slope_values, c="b")
plt.xlabel("c")
plt.ylabel("slope")

plt.show()

###############################
### DO NOT CHANGE THIS CELL ###
###############################
base = np.arange(0.001, 1, 0.001).reshape(-1, 1)
c = 0.5
X = base + np.random.normal(loc=[0], scale=c, size=base.shape)
y = 5 * base + np.random.normal(loc=[0], scale=c, size=base.shape)

fig = plt.figure()
plt.scatter(X, y)
plt.xlabel("x")
plt.ylabel("y")

plt.show()

###############################
### DO NOT CHANGE THIS CELL ###
###############################
pca_slope_values = []
linreg_slope_values = []
c_values = []
s_idx = 0

for i in range(30):
    for c in np.arange(0, 1.05, 0.05):
        # Calculate pca_slope_value (psv) and lr_slope_value (lsv), notice x_noise = True
        psv, lsv = Slope.addNoise(c, x_noise=True, seed=s_idx)

        # Append pca and lr slope values to list for plot function
        pca_slope_values.append(psv)
        linreg_slope_values.append(lsv)

        # Append c value to list for plot function
        c_values.append(c)

        # Increment random seed index
        s_idx += 1

fig = plt.figure()
plt.scatter(c_values, pca_slope_values, c="r")
plt.scatter(c_values, linreg_slope_values, c="b")
plt.xlabel("c")
plt.ylabel("slope")

plt.show()

###############################
### DO NOT CHANGE THIS CELL ###
###############################

from feature_reduction import FeatureReduction

###############################
### DO NOT CHANGE THIS CELL ###
###############################
bc_dataset = load_breast_cancer()
bc = pd.DataFrame(bc_dataset.data, columns=bc_dataset.feature_names)
print("Dataset Features: ", bc.columns.tolist())
bc["Diagnosis"] = bc_dataset.target

X = bc.drop("Diagnosis", axis=1)
y = bc["Diagnosis"]
featureselection = FeatureReduction()
# Run the functions to make sure two dictonaries are generated, one for each method
forward_selection_feature_map = FeatureReduction.forward_selection(
    X, y, [0.01, 0.1, 0.2]
)
backward_selection_feature_map = FeatureReduction.backward_elimination(
    X, y, [0.01, 0.1, 0.2]
)

Dataset Features:  ['mean radius', 'mean texture', 'mean perimeter', 'mean area', 'mean smoothness', 'mean compactness', 'mean concavity', 'mean concave points', 'mean symmetry', 'mean fractal dimension', 'radius error', 'texture error', 'perimeter error', 'area error', 'smoothness error', 'compactness error', 'concavity error', 'concave points error', 'symmetry error', 'fractal dimension error', 'worst radius', 'worst texture', 'worst perimeter', 'worst area', 'worst smoothness', 'worst compactness', 'worst concavity', 'worst concave points', 'worst symmetry', 'worst fractal dimension']

# evalute best p_value
# The evaluate features method performs linear regression on each specificed feature list.
# It calculates the rmse for each feature list and then displays the best significance level

FeatureReduction.evaluate_features(X, y, forward_selection_feature_map)
FeatureReduction.evaluate_features(X, y, backward_selection_feature_map)

significance level: 0.01, Removed features: {'area error', 'mean texture', 'mean radius', 'worst concavity', 'worst compactness', 'concavity error', 'mean perimeter', 'perimeter error', 'mean concavity', 'symmetry error', 'mean fractal dimension', 'worst perimeter', 'worst smoothness', 'mean symmetry', 'mean area', 'texture error', 'concave points error', 'fractal dimension error', 'mean smoothness'}
significance level: 0.01, RMSE: 0.23191025244141578
significance level: 0.1, Removed features: {'mean fractal dimension', 'mean symmetry', 'mean texture', 'mean perimeter', 'mean radius', 'mean area', 'worst compactness', 'texture error', 'concave points error', 'perimeter error', 'worst perimeter', 'worst smoothness', 'fractal dimension error', 'mean concavity', 'symmetry error', 'mean smoothness'}
significance level: 0.1, RMSE: 0.23821726582785402
significance level: 0.2, Removed features: {'mean fractal dimension', 'mean symmetry', 'mean texture', 'mean perimeter', 'mean radius', 'mean area', 'worst compactness', 'texture error', 'perimeter error', 'worst perimeter', 'worst smoothness', 'fractal dimension error', 'symmetry error', 'mean smoothness'}
significance level: 0.2, RMSE: 0.2412375366022595
Best significance level: 0.01, RMSE: 0.23191025244141578

significance level: 0.01, Removed features: {'area error', 'mean texture', 'mean radius', 'worst compactness', 'mean perimeter', 'worst concave points', 'perimeter error', 'compactness error', 'mean concavity', 'symmetry error', 'mean fractal dimension', 'worst perimeter', 'worst smoothness', 'mean symmetry', 'mean area', 'texture error', 'concave points error', 'fractal dimension error', 'mean smoothness'}
significance level: 0.01, RMSE: 0.24042471216353772
significance level: 0.1, Removed features: {'area error', 'mean fractal dimension', 'mean symmetry', 'mean texture', 'mean perimeter', 'mean area', 'worst compactness', 'worst concave points', 'texture error', 'perimeter error', 'worst perimeter', 'worst smoothness', 'fractal dimension error', 'compactness error', 'mean concavity', 'symmetry error', 'mean smoothness'}
significance level: 0.1, RMSE: 0.2374589600443534
significance level: 0.2, Removed features: {'area error', 'mean fractal dimension', 'mean symmetry', 'mean texture', 'mean perimeter', 'mean area', 'worst compactness', 'worst concave points', 'texture error', 'perimeter error', 'worst perimeter', 'worst smoothness', 'fractal dimension error', 'compactness error', 'symmetry error', 'mean smoothness'}
significance level: 0.2, RMSE: 0.2363767160772516
Best significance level: 0.2, RMSE: 0.2363767160772516

###############################
### DO NOT CHANGE THIS CELL ###
###############################

from regression import Regression
from svd_recommender import SVDRecommender

###############################
### DO NOT CHANGE THIS CELL ###
###############################

recommender = SVDRecommender()
recommender.load_movie_data()
regression = Regression()
# Read the data into the respective train and test dataframes
train, test = recommender.load_ratings_datasets()
print("---------------------------------------------")
print("Train Dataset Stats:")
print("Shape of train dataset: {}".format(train.shape))
print("Number of unique users (train): {}".format(train["userId"].unique().shape[0]))
print("Number of unique users (train): {}".format(train["movieId"].unique().shape[0]))
print("Sample of Train Dataset:")
print("------------------------------------------")
print(train.head())
print("------------------------------------------")
print("Test Dataset Stats:")
print("Shape of test dataset: {}".format(test.shape))
print("Number of unique users (test): {}".format(test["userId"].unique().shape[0]))
print("Number of unique users (test): {}".format(test["movieId"].unique().shape[0]))
print("Sample of Test Dataset:")
print("------------------------------------------")
print(test.head())
print("------------------------------------------")

# We will first convert our dataframe into a matrix of Ratings: R
# R[i][j] will indicate rating for movie:(j) provided by user:(i)
# users_index, movies_index will store the mapping between array indices and actual userId / movieId
R, users_index, movies_index = recommender.create_ratings_matrix(train)
print("Shape of Ratings Matrix (R): {}".format(R.shape))

# Replacing `nan` with average rating given for the movie by all users
# Additionally, zero-centering the array to perform SVD
mask = np.isnan(R)
masked_array = np.ma.masked_array(R, mask)
r_means = np.array(np.mean(masked_array, axis=0))
R_filled = masked_array.filled(r_means)
R_filled = R_filled - r_means

---------------------------------------------
Train Dataset Stats:
Shape of train dataset: (88940, 4)
Number of unique users (train): 671
Number of unique users (train): 8370
Sample of Train Dataset:
------------------------------------------
   userId  movieId  rating   timestamp
0       1     2294     2.0  1260759108
1       1     2455     2.5  1260759113
2       1     3671     3.0  1260759117
3       1     1339     3.5  1260759125
4       1     1343     2.0  1260759131
------------------------------------------
Test Dataset Stats:
Shape of test dataset: (10393, 4)
Number of unique users (test): 671
Number of unique users (test): 4368
Sample of Test Dataset:
------------------------------------------
   userId  movieId  rating   timestamp
0       1     2968     1.0  1260759200
1       1     1405     1.0  1260759203
2       1     1172     4.0  1260759205
3       2       52     3.0   835356031
4       2      314     4.0   835356044
------------------------------------------
Shape of Ratings Matrix (R): (671, 8370)

###############################
### DO NOT CHANGE THIS CELL ###
###############################

from utilities.localtests import TestSVDRecommender

unittest_svd_rec = TestSVDRecommender()
unittest_svd_rec.test_recommender_svd()

UnitTest passed successfully for "recommender_svd() function"!

###############################
### DO NOT CHANGE THIS CELL ###
###############################

# Implement the method `recommender_svd` and run it for the following values of features
no_of_features = [2, 3, 8, 15, 18, 25, 30]
test_errors = []

for k in no_of_features:
    U_k, V_k = recommender.recommender_svd(R_filled, k)
    pred = []  # to store the predicted ratings
    for _, row in test.iterrows():
        user = row["userId"]
        movie = row["movieId"]
        u_index = users_index[user]
        # If we have a prediction for this movie, use that
        if movie in movies_index:
            m_index = movies_index[movie]
            pred_rating = np.dot(U_k[u_index, :], V_k[:, m_index]) + r_means[m_index]
        # Else, use an average of the users ratings
        else:
            pred_rating = np.mean(np.dot(U_k[u_index], V_k)) + r_means[m_index]
        pred.append(pred_rating)
    test_error = regression.rmse(test["rating"], pred)
    test_errors.append(test_error)
    print("RMSE for k = {} --> {}".format(k, test_error))

RMSE for k = 2 --> 1.0223035413708281
RMSE for k = 3 --> 1.022526649417955
RMSE for k = 8 --> 1.0182709203352787
RMSE for k = 15 --> 1.017307118738714
RMSE for k = 18 --> 1.0166562048687973
RMSE for k = 25 --> 1.0182856984912254
RMSE for k = 30 --> 1.0186282488126601

###############################
### DO NOT CHANGE THIS CELL ###
###############################
fig = plt.figure()
plt.plot(no_of_features, test_errors, "bo")
plt.plot(no_of_features, test_errors)
plt.xlabel("Value for k")
plt.ylabel("RMSE on Test Dataset")
plt.title("SVD Recommendation Test Error with Different k values")

plt.show()

###############################
### DO NOT CHANGE THIS CELL ###
###############################

unittest_svd_rec.test_predict()

Top 3 Movies the User would want to watch:
Captain America: The First Avenger (2011)
Ant-Man (2015)
Avengers: Age of Ultron (2015)
--------------------------------------------------------------
UnitTest passed successfully for "predict() function"!

###############################
### DO NOT CHANGE THIS CELL ###
###############################

marvel_movies = [
    "Thor: The Dark World (2013)",
    "Avengers: Age of Ultron (2015)",
    "Ant-Man (2015)",
    "Iron Man 2 (2010)",
    "Avengers, The (2012)",
    "Thor (2011)",
    "Captain America: The First Avenger (2011)",
]
marvel_labels = ["Blue"] * len(marvel_movies)
star_wars_movies = [
    "Star Wars: Episode IV - A New Hope (1977)",
    "Star Wars: Episode V - The Empire Strikes Back (1980)",
    "Star Wars: Episode VI - Return of the Jedi (1983)",
    "Star Wars: Episode I - The Phantom Menace (1999)",
    "Star Wars: Episode II - Attack of the Clones (2002)",
    "Star Wars: Episode III - Revenge of the Sith (2005)",
]
star_wars_labels = ["Green"] * len(star_wars_movies)


movie_titles = star_wars_movies + marvel_movies
genre_labels = star_wars_labels + marvel_labels

movie_indices = [
    movies_index[recommender.get_movie_id_by_name(str(x))] for x in movie_titles
]

_, V_k = recommender.recommender_svd(R_filled, k=2)
x, y = V_k[0, movie_indices], V_k[1, movie_indices]
fig = plt.figure()
plt.scatter(x, y, c=genre_labels)
for i, movie_name in enumerate(movie_titles):
    plt.annotate(movie_name, (x[i], y[i]))

Spring 2024 CS4641/CS7641 Homework 3¶

Instructor: Dr. Mahdi Roozbahani¶

Deadline: Friday, March 29th, 11:59 pm EST¶

Instructions for the assignment¶

Using the autograder¶

Using the local tests ¶

Deliverables and Points Distribution¶

Q1: Image Compression [30pts]¶

Deliverables: imgcompression.py and printed results¶

Q2: Understanding PCA [20pts]¶

Deliverables: pca.py and written portion¶

Q3: Regression and Regularization [80pts: 50pts + 20pts Grad / 6% Bonus for Undergrads + 2.3% Bonus for All]¶

Deliverables: regression.py and Written portion¶

Q4: Naive Bayes and Logistic Regression [35pts]¶

Deliverables: logistic_regression.py and Written portion¶

Q5: Noise in PCA and Linear Regression [15pts]¶

Deliverables: Written portion¶

Q6: Feature Reduction.py [5.6% Bonus for All]¶

Deliverables: feature_reduction.py and Written portion¶

Q7: Movie Recommendation with SVD [2.1% Bonus for All]¶

Deliverables: svd_recommender.py and Written portion¶

Submission Instructions¶

0 Set up¶

Library imports¶

Q1: Image Compression [30 pts] [P] | [W]¶

Load images data and plot¶

1.1 Image compression [20pts] [P]¶

1.1.1 Local Tests for Image Compression Black and White Case [No Points]¶

1.1.2 Local Tests for Image Compression Color Case [No Points]¶

1.2.1 Black and white [5 pts] [W]¶

1.2.2 Black and White Compression Savings [No Points]¶

1.3.1 Color image [5 pts] [W]¶

1.3.2 Color Compression Savings [No Points]¶

Q2: Understanding PCA [20 pts] [P] | [W]¶

2.1 Implementation [10 pts] [P]¶

2.1.1 Local Tests for PCA [No Points]¶

2.2 Visualize [5 pts] [W]¶

Iris Dataset¶

2.3 PCA Reduced Facemask Dataset Analysis [5 pts] [W]¶

Facemask Dataset¶

2.4 PCA Exploration [No Points]¶

Emotion Dataset [No Points]¶

Q3 Polynomial regression and regularization [80pts: 50pts + 20pts Grad / 1.5% Bonus for Undergrads + 2.3% Bonus for All] [P] | [W]¶

3.1 Regression and regularization implementations [50pts: 30 pts + 20 Grad / 1.5% Bonus for Undergrad] [P]¶

3.1.1 Local Tests for Helper Regression Functions [No Points]¶

3.1.2 Local Tests for Linear Regression Functions [No Points]¶

3.1.3 Local Tests for Ridge Regression Functions [No Points]¶

3.1.4 Local Tests for Gradient Descent and SGD (Bonus for Undergrad Tests) [No Points]¶

3.2 About RMSE [3 pts] [W]¶

3.3 Testing: General Functions and Linear Regression [5 pts] [W]¶

Linear Closed 10 Samples¶

3.4 Testing: Testing ridge regression [5 pts] [W]¶

3.4.1 [3pts] [W]¶

Linear vs. Ridge Regression¶

3.4.2 Why does ridge regression achieve a lower RMSE than linear regression on 10 sample points? [1pts] [W]¶

3.4.3 Describe and contrast two scenarios (real life applications): One where linear is more suitable than ridge, and one in which ridge is better choice than linear. Explain why. [1 pts] [W]¶

3.4.4 What is the impact of having some highly correlated features on the data set in terms of linear algebra? Mathematically explain (include expressions) how ridge has an advantage on this in comparison to linear regression. Include the idea of numerical stability. [2pts] [W]¶

3.5 Cross validation [7 pts] [W]¶

3.6 Noisy Input Samples in Linear Regression [2.3% Bonus for All] [W]¶

Q4: Naive Bayes and Logistic Regression [35pts] [P] | [W]¶

Training Naive Bayes¶

Testing Naive Bayes¶

4.1 Profile Screening [7pts] [W]¶

4.1.1 Profile Screening using Naive Bayes [5pts][W]¶

4.1.2 AI-Driven Profile Screening[2pts] [W]¶

4.2 News Data Sentiment Classification via Logistic Regression [30pts] [P]¶

Logistic Regression Overview:¶

4.2.1 Local Tests for Logistic Regression [No Points]¶

4.2.2 Logistic Regression Model Training [No Points]¶

4.2.3 Logistic Regression Model Evaluation [No Points]¶

Q5: Noise in PCA and Linear Regression [15pts] [W]¶

5.1 Slope Functions [5 pts] [W]¶

5.2 Analysis Setup [5 pts] [W]¶

Error in y¶

Error in $x$ and $y$¶

5.3. Analysis [5 pts] [W]¶

Q6 Feature Reduction Implementation [5.6% Bonus for All] [P] | [W]¶

6.1 Implementation [4%] [P]¶

Forward Selection:¶

Backward Elimination:¶

7.1.1 Implement the `recommender_svd` method to use SVD for Recommendation [1.05%] [P]¶

Local Test for `recommender_svd` Function [No Points]¶

7.1.2 Implement the `predict` method to find which movie a user is interested in watching next [1.05%] [P]¶

Local Test for `predict` Functions [No Points]¶